package com.coolpad.job.device.service.spider;

import com.coolpad.job.device.model.CityNodeBO;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.http.converter.StringHttpMessageConverter;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import org.springframework.web.client.RestTemplate;

import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;

/**
 * ????:????<br/>
 *
 * @Author:shengtao_yang
 * @Date: 2020/12/22 18:21
 * @since: 1.0.0
 */
@Slf4j
@Service
public class CityPartitionSpider {

    public final String base_url = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/index.html";

    @Autowired
    private RestTemplate restTemplate;


    public List<CityNodeBO> crawl(){
        List<CityNodeBO> provincesList = new ArrayList<>();
        Elements provinceElements = getNodeElement(base_url, "provincetr","a");
        provinceElements.forEach(provinceElement -> {
            CityNodeBO provinceNode = new CityNodeBO("", provinceElement.text());
            provincesList.add(provinceNode);
            List<CityNodeBO> cityNodes = new ArrayList<>();
            provinceNode.setSub(cityNodes);
            String city_href = buildUrl(base_url, provinceElement.attr("href"));
            Elements citTrElements;
            citTrElements = retryIfException(city_href, "citytr",3);
            if(citTrElements == null){
                log.error("province:{}, city:{}, count:{}, town?{}",provinceElement.text());
                return;
            }
            citTrElements.forEach(cityTrElement -> {
                Elements cityTds = cityTrElement.getElementsByTag("td");
                provinceNode.setCode("10" + cityTds.get(0).text().substring(0,3));
                CityNodeBO cityNode = new CityNodeBO(cityTds.get(0).text(), cityTds.get(1).text());
                cityNodes.add(cityNode);
                List<CityNodeBO> countyNodes = new ArrayList<>();
                cityNode.setSub(countyNodes);
                String county_href = buildUrl(city_href,cityTds.select("a").attr("href"));
                Elements countyTrElements ;
                countyTrElements = retryIfException(county_href, "countytr",3);
                if(countyTrElements == null){
                    log.error("province:{}, city:{}, count:{}, town?{}",provinceElement.text(),cityTds.get(1).text());
                    return;
                }
                countyTrElements.forEach(countyTrElement -> {
                    Elements countyTds = countyTrElement.getElementsByTag("td");
                    CityNodeBO countyNode = new CityNodeBO(countyTds.get(0).text(), countyTds.get(1).text());
                    String town_sub_url = countyTds.select("a").attr("href");
                    if(StringUtils.isEmpty(town_sub_url)) return;
                    countyNodes.add(countyNode);
                    List<CityNodeBO> townNodes = new ArrayList<>();
                    countyNode.setSub(townNodes);
                    String town_href = buildUrl(county_href,town_sub_url);
                    Elements townTrElements;
                    townTrElements = retryIfException(town_href,"towntr",3);
                    if(townTrElements == null){
                        log.error("province:{}, city:{}, count:{}, town?{}",provinceElement.text(),cityTds.get(1).text(),countyTds.get(1).text());
                        return;
                    }
                    townTrElements.forEach(townTrElement -> {
                        Elements townTds = townTrElement.getElementsByTag("td");
                        CityNodeBO townNode = new CityNodeBO(townTds.get(0).text(), townTds.get(1).text());
                        townNodes.add(townNode);
                    });
                });
            });
            log.info("crawl {} end",provinceElement.text());
        });
        return provincesList;
    }

    private String buildUrl(String current_url,String sub_url){
        return current_url.substring(0,current_url.lastIndexOf("/"))  + "/" + sub_url;
    }

    private Elements retryIfException(String url, String className, int time){
        int count = 1;
        boolean retry = false;
        do {
            try {
                Elements nodeElement = getNodeElement(url, className);
                if(retry){
                    log.info("retry success:{}",url);
                }
                return nodeElement;
            } catch (Exception e) {
                log.error("retry fail count?{}", count++);
                retry = true;
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException e1) {

                }
            }
        }while (count <= time);
        return null;
    }

    private Elements getNodeElement(String url, String className, String tagName){
        restTemplate.getMessageConverters().set(1,new StringHttpMessageConverter(Charset.forName("GBK")));

        ResponseEntity<String> response = restTemplate.getForEntity(url, String.class);
        Document doc = Jsoup.parse(response.getBody());
        Elements classEles = doc.getElementsByClass(className);
        Elements nodes = new Elements();
        classEles.forEach(classEle -> {
            Elements a = classEle.getElementsByTag(tagName);
            nodes.addAll(a);
        });
        return nodes;
    }

    private Elements getNodeElement(String url, String className){
        restTemplate.getMessageConverters().set(1,new StringHttpMessageConverter(Charset.forName("GBK")));
        ResponseEntity<String> response = restTemplate.getForEntity(url, String.class);
        Document doc = Jsoup.parse(response.getBody());
        Elements classEles = doc.getElementsByClass(className);
        return classEles;
    }
}
